diff --git a/sklearn/calibration.py b/sklearn/calibration.py
index 2c4a33616..1ec6df9bd 100644
--- a/sklearn/calibration.py
+++ b/sklearn/calibration.py
@@ -308,42 +308,97 @@ class CalibratedClassifierCV(ClassifierMixin, MetaEstimatorMixin, BaseEstimator)
         if sample_weight is not None:
             sample_weight = _check_sample_weight(sample_weight, X)
 
-        for sample_aligned_params in fit_params.values():
-            check_consistent_length(y, sample_aligned_params)
+        for key, value in fit_params.items():
+            if key == 'eval_set':
+                # Special handling for eval_set to avoid length check
+                continue
+            check_consistent_length(y, value)
+
+        # Set `classes_` using all `y`
+        label_encoder_ = LabelEncoder().fit(y)
+        self.classes_ = label_encoder_.classes_
+        n_classes = len(self.classes_)
+
+        # sample_weight checks
+        fit_parameters = signature(estimator.fit).parameters
+        supports_sw = "sample_weight" in fit_parameters
+        if sample_weight is not None and not supports_sw:
+            estimator_name = type(estimator).__name__
+            warnings.warn(
+                f"Since {estimator_name} does not appear to accept sample_weight, "
+                "sample weights will only be used for the calibration itself. This "
+                "can be caused by a limitation of the current scikit-learn API. "
+                "See the following issue for more details: "
+                "https://github.com/scikit-learn/scikit-learn/issues/21134. Be "
+                "warned that the result of the calibration is likely to be "
+                "incorrect."
+            )
 
-        # TODO(1.4): Remove when base_estimator is removed
-        if self.base_estimator != "deprecated":
-            if self.estimator is not None:
-                raise ValueError(
-                    "Both `base_estimator` and `estimator` are set. Only set "
-                    "`estimator` since `base_estimator` is deprecated."
+        # Check that each cross-validation fold can have at least one
+        # example per class
+        if isinstance(self.cv, int):
+            n_folds = self.cv
+        elif hasattr(self.cv, "n_splits"):
+            n_folds = self.cv.n_splits
+        else:
+            n_folds = None
+        if n_folds and np.any(
+            [np.sum(y == class_) < n_folds for class_ in self.classes_]
+        ):
+            raise ValueError(
+                f"Requesting {n_folds}-fold "
+                "cross-validation but provided less than "
+                f"{n_folds} examples for at least one class."
+            )
+        cv = check_cv(self.cv, y, classifier=True)
+
+        if self.ensemble:
+            parallel = Parallel(n_jobs=self.n_jobs)
+            self.calibrated_classifiers_ = parallel(
+                delayed(_fit_classifier_calibrator_pair)(
+                    clone(estimator),
+                    X,
+                    y,
+                    train=train,
+                    test=test,
+                    method=self.method,
+                    classes=self.classes_,
+                    supports_sw=supports_sw,
+                    sample_weight=sample_weight,
+                    **fit_params,
                 )
-            warnings.warn(
-                "`base_estimator` was renamed to `estimator` in version 1.2 and "
-                "will be removed in 1.4.",
-                FutureWarning,
+                for train, test in cv.split(X, y)
             )
-            estimator = self.base_estimator
         else:
-            estimator = self.estimator
-
-        if estimator is None:
-            # we want all classifiers that don't expose a random_state
-            # to be deterministic (and we don't want to expose this one).
-            estimator = LinearSVC(random_state=0)
-
-        self.calibrated_classifiers_ = []
-        if self.cv == "prefit":
-            # `classes_` should be consistent with that of estimator
-            check_is_fitted(self.estimator, attributes=["classes_"])
-            self.classes_ = self.estimator.classes_
-
-            pred_method, method_name = _get_prediction_method(estimator)
-            n_classes = len(self.classes_)
-            predictions = _compute_predictions(pred_method, method_name, X, n_classes)
+            this_estimator = clone(estimator)
+            _, method_name = _get_prediction_method(this_estimator)
+            fit_params = (
+                {"sample_weight": sample_weight}
+                if sample_weight is not None and supports_sw
+                else None
+            )
+            pred_method = partial(
+                cross_val_predict,
+                estimator=this_estimator,
+                X=X,
+                y=y,
+                cv=cv,
+                method=method_name,
+                n_jobs=self.n_jobs,
+                fit_params=fit_params,
+            )
+            predictions = _compute_predictions(
+                pred_method, method_name, X, n_classes
+            )
 
+            if sample_weight is not None and supports_sw:
+                this_estimator.fit(X, y, sample_weight=sample_weight)
+            else:
+                this_estimator.fit(X, y)
+            # Note: Here we don't pass on fit_params because the supported
+            # calibrators don't support fit_params anyway
             calibrated_classifier = _fit_calibrator(
-                estimator,
+                this_estimator,
                 predictions,
                 y,
                 self.classes_,
